#! /usr/bin/env python
#coding=utf-8
#******************************************************************************
# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
# licensed under the Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#     http://license.coscl.org.cn/MulanPSL2
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
# PURPOSE.
# See the Mulan PSL v2 for more details.
# Author: senlin
# Create: 2020-08-10
# ******************************************************************************/

#======================example for showing
#print(inputspec.version)  # 3.8.0
#print(inputspec.sources[0])  # http://llvm.org/releases/%{version}/%{name}-%{version}.src.tar.xz
#print(replace_macros(inputspec.sources[0], inputspec))  # http://llvm.org/releases/3.8.0/llvm-3.8.0.src.tar.xz

#     __          __     __             ___     __      __     __
#   /     |  /  |      /     |  /     |       |    \  |      /
#  |      | /   | __  |      | /      | __    | __ /  | __  |
#  |      | \   |     |      | \           |  |       |     |
#   \ __  |  \  | __   \ __  |  \      ___ |  |       | __   \ __

import sys
import os
import requests
import argparse
import configparser
import re
import hashlib
import tarfile
import bz2
import shutil
import chardet
import logging
import subprocess
import urllib
import tempfile
import traceback
import zipfile
import tldextract

from    spec import Spec, replace_macros
from    LoggerWithDate import Logger

DEFAULT_LICENSE_DICTIONARY_PATH = "./license_translations"   
DEFAULT_LICENSE_LIST_PATH       = "./LicenseList.txt"
DEFAULT_KEYWORDS_PATH           = "./KEYWORDS" # keywords
DEFAULT_OK_PKGS_PATH            = "ok_pkgs"

UNUSED_FILE_SUFFIX              = (".spec~", ".specfc", ".spec.old", ".osc")
COMPRESSED_TYPE                 = (".tar.gz", ".tgz", ".gz", ".zip", ".bz2")

src_file_name_list        =    []
keywords                  =    []
OK_pkg_list               =    []
NOTOK_pkg_list            =    []

licenses_for_source_files =    []
licenses_for_spec         =    []
license_translations      =    {}
license_list              =    {}

NON_SENSITIVE_WORDS       = ["redhat.com", "fedoraproject.org"] # 如果想忽略带有keywords的特殊匹配项，在这里添加，正式开源后会清除这里的配置
LICENSE_FILE_TARGET                   = ["apache-2.0",
                                          "artistic",
                                          "artistic.txt",
                                          "libcurllicense",
                                          "gpl.txt",
                                          "gpl2.txt",
                                          "gplv2.txt",
                                          "notice",
                                          "about_bsd.txt",
                                          "mit",
                                          "pom.xml",
                                          "meta.yml"]
LICENSE_TARGET_PAT  = re.compile(r"^(copying)|(copyright)|(copyrights)|(licenses)|(licen[cs]e)(\.(txt|xml))?$")

log_check   =  Logger('CheckSpec.py').getlog()                        # 获取一个日志对象，这里参数 只有模块名

def main():
    """ Entry point for check_licenses."""
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--tars_path", default=None, nargs="?", required=True,
                        help="set the path of all pkgs.")

                        
    parser.add_argument("-w", "--writespec", dest="writespec", action="store_true",
                        default=False,
                        help="Overwrite the licenses of SPEC file")
                        
    parser.add_argument("-d", "--downloadpath", default="/home/PkgSource/", nargs="?",
                        help="The dest download or extract path of tarball"
                        " (e.g. /home/tmp_tarball default: /home/PkgSource/)")

    parser.add_argument("--specshell", default="",
                        help="call shell to change spec file")

    parser.add_argument("--specify", action="store_true", default=False,
                        help="only check the file in specified software package")

    args = parser.parse_args()
    
    if load_config(args) == False:                                                          # 如果加载配置失败，退出程序
        log_check.warning("Something is wrong, please check config")
        sys.exit(0)
    log_check.info("Load configuration files successfully!")
    process(args)


def load_config(args):
    """
    清空之前生成的临时日志，导入关键词、license黑白名单
    """
    res_flag = True
    if args.tars_path:
        currentPath     = os.path.dirname(os.path.abspath(__file__))  # return current path
        ok_pkgs_path    = os.path.join(currentPath, DEFAULT_OK_PKGS_PATH)
        if not os.path.exists(ok_pkgs_path):
            os.makedirs(ok_pkgs_path, exist_ok=True)
    else:
        log_check.warning("Something is wrong, please check parameter: -p")
        sys.exit(0)
        

    if os.path.exists(args.downloadpath):
        shutil.rmtree(args.downloadpath)
    os.makedirs(args.downloadpath, exist_ok=True)
    
    count_key_word = 0
    subprocess.run("rm -rf /var/tmp/*same_lic_tar.out", shell=True)
    log_check.info("Beginning loading configuration files:")

    if load_key_words(keywords, DEFAULT_KEYWORDS_PATH) == False:
        log_check.warning("load %s failed", DEFAULT_KEYWORDS_PATH)
        return False
        
    if load_licenses_list(license_list, DEFAULT_LICENSE_LIST_PATH) == False:
        log_check.warning("load %s failed", DEFAULT_LICENSE_LIST_PATH)
        return False
    
    if load_licenses_translate(DEFAULT_LICENSE_DICTIONARY_PATH) == False:                            # 初始化license字典
        log_check.warning("load %s failed", DEFAULT_LICENSE_DICTIONARY_PATH)
        return False


def load_key_words(words, filename):
    """
    从关键词文件中导入关键词至keywords全局变量
    :return: num of keywords
    """
    conf_dir = os.path.dirname(os.path.abspath(__file__))
    conf_path = os.path.join(conf_dir, filename)
    if not os.path.isfile(conf_path):
        log_check.warning("not found the config file: %s", conf_path)
        return False
    with open(conf_path, "r") as conf_file:
        for line in conf_file:
            words.append(line.strip())
    return True


def load_licenses_list(licenses, filename):
    """
    Read the licenseslist file.
    """
    conf_dir = os.path.dirname(os.path.abspath(__file__))
    conf_path = os.path.join(conf_dir, filename)
    if not os.path.isfile(conf_path):
        log_check.warning("not found the white&black license file: %s", conf_path)
        return False
    with open(conf_path, "r") as conf_file:
        for line in conf_file:
            key_name, final_name = line.rsplit(", ", 1)
            licenses[key_name] = final_name.rstrip()
    return True
    

def load_licenses_translate(filename):
    """
    Read the licenses_translate file.
    """
    conf_dir = os.path.dirname(os.path.abspath(__file__))
    conf_path = os.path.join(conf_dir, filename)
    if not os.path.isfile(conf_path):
        log_check.info("not found the config file: %s", conf_path)
        return False
    with open(conf_path, "r") as conf_file:
        for line in conf_file:
            if line.startswith("#"):
                continue
            key_name, final_name = line.rsplit(", ", 1)
            license_translations[key_name] = final_name.rstrip()
    return True


def process(args):
    """ Preparation and processing of the main entrance."""
    truncate_file("NOTOK_pkg_list.txt")
    truncate_file("OK_pkg_list.txt")

    if args.specify:
        process_specified_pkg(args)
    else:
        process_all_pkgs(args)
    record_pkg_List()
    process_log(log_check.logName)
    log_check.info("process all of packages end!")


def process_specified_pkg(args):
    """
    Process inspection actions for specified packages.
    """
    args.tars_path = os.path.abspath(args.tars_path)
    package_name = os.path.basename(args.tars_path)   # 获取路径的最后一层目录
    if not os.path.isdir(args.tars_path):
        log_check.error('Illegal path, is not a directory: {tar_path}'.format(tar_path=args.tars_path))
        return 
    if not package_name.startswith('.'):
        res_flag = process_pkg(args, args.tars_path, package_name)
        if res_flag == True:
            OK_pkg_list.append(package_name)
            currentPath = os.path.dirname(os.path.abspath(__file__))  # return current path
            new_path    = os.path.join(currentPath, DEFAULT_OK_PKGS_PATH)
            shutil.move(args.tars_path, new_path)
            log_check.info('///////////////////End process package: %s, result is OK!', package_name)
        else:
            NOTOK_pkg_list.append(package_name)
            log_check.error('///////////////////End process package: %s, result is NOT OK!', package_name)


def process_all_pkgs(args):
    """
    Process inspection actions for all packages.
    """
    args.tars_path = os.path.abspath(args.tars_path)
    with os.scandir(args.tars_path) as it:
        for entry in it:
            if not entry.name.startswith('.'):
                pkgPath = os.path.join(args.tars_path, entry.name)
                package_name = entry.name
                res_flag = process_pkg(args, pkgPath, package_name)           # 处理每一个软件包
                if res_flag == True:
                    OK_pkg_list.append(package_name)

                    currentPath = os.path.dirname(os.path.abspath(__file__))  # return current path
                    new_path    = os.path.join(currentPath, DEFAULT_OK_PKGS_PATH)

                    shutil.move(pkgPath, new_path)
                    log_check.info('///////////////////End process package: %s, result is OK!', package_name)
                else:
                    NOTOK_pkg_list.append(package_name)
                    log_check.error('///////////////////End process package: %s, result is NOT OK!', package_name)


def process_pkg(args, pkgPath, package_name):
    """
    Process inspection actions for package.
    """
    log_check.info('///////////////////Start process package: %s ///////////////////', package_name)
    res_flag            = True
    spec_name           = ""
    compressed_files    = []
    
    if args.specshell != "":
         change_spec_by_shell(args.specshell, pkgPath)                            # 在check spec格式前调用脚本修改spec
         
    # 1.解析并校验URL、source
    for file_name in os.listdir(pkgPath):
        filePath = os.path.join(pkgPath, file_name)                                  # 加上路径，dirpath是遍历时文件对应的路径
        log_check.info('Scanning file: %s', filePath)
        if is_unused_spec_file(file_name):                                          # 使用脚本生成的临时文件不进行检查
            continue
        if is_compressed_file(filePath):                                            # 记录哪些是压缩包文件以备解析license
            compressed_files.append(filePath)
            continue
        if file_name.endswith(".spec"):
            spec_name = filePath
            inputspec = Spec.from_file(filePath)                                    # 获取spec初步解析之后的对象
            if not check_url_of_pkg(inputspec):                                  # 校验URL字段的有效性
                res_flag = False
            if not check_url_of_source(pkgPath, inputspec, args.downloadpath):                    # 校验source的链接的有效性及md5值
                res_flag = False
            if not check_url_source_consistency(inputspec):
                res_flag = False
        
        line_list = find_keyword_in_file(filePath, file_name, False)
        if line_list:
            for line_info in line_list:
                # 修改日志输出级别
                log_check.error("[{filename}:{no}] {line} | keywords: {keywords}".format(filename=file_name, 
                                                                                  no=line_info["line_no"], 
                                                                                  line=line_info["line"],
                                                                                  keywords=line_info["keywords"]))
            res_flag = False
        else:
            log_check.info('Check keywords in: %s, PASS!', file_name)

    # 2.解析并校验license
    scan_licenses_in_SPEC(spec_name)                                                # 对spec中license的解析只需要一次
    if check_license_safe(licenses_for_spec) == False:
        res_flag = False
        
    for tname in compressed_files:
        tmp_file = os.path.join(pkgPath, tname)
        temp_extract_path = os.path.abspath(tempfile.mkdtemp(dir=args.downloadpath)) # 创建临时目录以备解压压缩文件到指定目录
        if tmp_file.endswith(("tar.gz", ".tgz")): # 当前source主要的压缩文件格式
            if extract_tar(tmp_file, temp_extract_path) == False:
                res_flag = False
                
        if tmp_file.endswith("tar.bz2"): # 当前source主要的压缩文件格式
            if extract_bz2(tmp_file, temp_extract_path) == False:
                res_flag = False
                
        if tmp_file.endswith(".zip"): # 当前source主要的压缩文件格式
            if extract_zip(tmp_file, temp_extract_path) == False:
                res_flag = False

        analyse_licenses_in_compressed_file(temp_extract_path)  # 解析压缩包形式的source文件里面的license
        log_check.info("all licenses from %s is: %s", tname, licenses_for_source_files)
        shutil.rmtree(temp_extract_path)

        if check_license_safe(licenses_for_source_files) == False:
            res_flag = False
        if check_licenses_is_same():
            log_check.info("licenses from source files are same as form SPEC:"
                        "%s == %s", licenses_for_source_files, licenses_for_spec)
        else:
            res_flag = False
            log_check.warning("licenses from source files are not same as form SPEC:"
                        "%s <=> %s", licenses_for_source_files, licenses_for_spec)
            
    licenses_for_source_files.clear()
    licenses_for_spec.clear()                                                       # 每一轮解析完当前组件的license之后，清空数组
    
    return res_flag


def is_compressed_file(src_file):
    """
    Determine whether it is a compressed file.
    """
    if os.path.isfile(src_file):
        if src_file.endswith(COMPRESSED_TYPE):
            return True
    return False
    
    
def analyse_licenses_in_compressed_file(extract_path):
    """
    Begin process licenses in source file. 
    """
    files = os.listdir(extract_path)
    extraction_folder_name = files[0]
    extracted_file_folder = os.path.join(extract_path, extraction_folder_name)
    scan_licenses_in_LICENSE(extracted_file_folder)                                 # 扫描解压目录里面的文件有可能携带的license


def scan_licenses_in_LICENSE(srcdir):
    """
    Find LICENSE files and scan. 
    """
    res_flag = True
    if not os.path.exists(srcdir):
        log_check.error("%s not exist.", srcdir)
        return False

    with os.scandir(srcdir) as it:
        for entry in it:
            if not entry.name.startswith('.'):
                file_name=entry.name
                if file_name.lower() in LICENSE_FILE_TARGET or LICENSE_TARGET_PAT.search(file_name.lower()):
                    log_check.info("scan the license target file: %s", file_name)
                    if entry.is_file():
                        scan_licenses(os.path.join(srcdir, file_name))
                    else:
                        res_flag = scan_licenses_in_LICENSE(os.path.join(srcdir, file_name))
    return res_flag


def check_license_safe(licenses):
    """
    Check if the license is in the blacklist. 
    """
    for lic in licenses:
        res = license_list.get(lic, "Need review")
        if res == "white":
            return True
        elif res == "black":
            log_check.error("This license: %s is not safe", lic)
            return False
        else: 
            log_check.warning("This license: %s need to be review", lic)
            return False


def is_include_non_sensitive_word(line_info):
    """
        全词匹配：判断输入的字符串中是否有关键字
    """
    for word in NON_SENSITIVE_WORDS:
        #先用find函数过滤
        if  -1 != line_info.find(word):
            #匹配下列情况：
            #^word$;
            #^word+任意非字母数字下划线;
            #任意非字母数字下划线)+word$;
            #任意非字母数字下划线+word+任意非字母数字下划线
            pattern_str='(^'+word+'$)|(^'+word+'(\W+).*)|(.*(\W+)'+word+'$)|(.*(\W+)'+word+'(\W+).*)'
            m = re.match(r''+pattern_str+'', line_info)
            if m:
                return True
    return False
    
    
def find_keyword_in_file(filepath, filename, save_code=False):
    """
    返回文件中所有包含keywords的行号、代码，数据结构如下
    [
        {
            line_no: xxx,
            line: xxx,
            keywords: ['a', 'b']
        },
    ]
    :return: list
    """
    line_list = []

    try:
        fileObj = open(filepath, "r")
        lines = fileObj.readlines()
        line_no = 0
        for line in lines:
            line_no += 1
            keyword_set = []
            for key in keywords:
                if key in line and is_include_non_sensitive_word(line) == False:
                    keyword_set.append(key)
            if len(keyword_set) > 0:
                line_info = {}
                line_info["keywords"] = keyword_set
                line_info["line_no"] = line_no
                line_info["line"] = line.strip() if save_code else ""
                line_list.append(line_info)
    except LookupError:  #获取异常类型（）
        # 异常发生的时候不会崩溃，而是执行这个里面的代码
        log_check.warning("%s can not be opened", filename)
        line_list = []
    except FileNotFoundError:
        log_check.error("%s is not found", filename)
        line_list = []
    finally:
        return line_list


def check_url_of_source(pkg_path, inputspec, download_path):
    """
    Check the validity of the download link pointing to 
    the source file and compare the md5 value after downloading. 
    """
    validFlag = True
    download_path = os.path.join(download_path, inputspec.name)                     # 指定下载文件存放的路径
    for source_index, source_url in inputspec.sources_dict.items(): 
        source_url = source_url.strip('\n')
        source_url = replace_macros(source_url, inputspec)                          # 替换网址中可能存在的宏
        log_check.info("%s: %s", source_index, source_url)

        if not os.path.exists(download_path):
            log_check.warning("Selected folder not exist, try to create it.")
            os.makedirs(download_path)
        
        log_check.info("Try downloading: %s", source_index)
        source_file_name = source_url.split('/')[-1]
        download_source_file = os.path.join(download_path, source_file_name)
        origin_source_file = os.path.join(pkg_path, source_file_name)               # 指定本地已有的source文件
        if os.path.exists(download_source_file):
            validFlag = check_md5(origin_source_file, download_source_file, source_index)         # 校验URL下载的文件的md5sum
        else:
            try:
                urllib.request.urlretrieve(source_url, filename=download_source_file)                             # 下载source文件
                validFlag = check_md5(origin_source_file, download_source_file, source_index)     # 校验URL下载的文件的md5sum
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info() 
                log_check.error("exc_type: %s, exc_value: %s",  exc_type, exc_value)
                validFlag = False
        
        if validFlag == True: 
            log_check.info('check url and md5 of %s PASS!', source_index)
        else:
            log_check.error('check url and md5 of %s Failed!', source_index)
                
    return validFlag


def check_url_source_consistency(inputspec):
    """
    检查spec中url和source0顶级域名的一致性
    """
    url_domain = tldextract.extract(inputspec.url).domain
    source0_url = inputspec.sources_dict.get("Source0", "")
    source0_domain = tldextract.extract(source0_url).domain
    if url_domain == source0_domain:
        log_check.info("check domain of url and source0 PASS!")
        return True
    log_check.warning("url domain({}) is different from source0 domain({})".format(url_domain, source0_domain))
    return False


def check_url_of_pkg(inputspec):
    """
    Check the validity of the URL. 
    """
    validFlag = True
    try:
        homepage = inputspec.url
        homepage = replace_macros(inputspec.url, inputspec)
        log_check.info('homepage: %s',  homepage)
        status = get_http_status_code(homepage.strip('\n'))
        if status != 200:
            log_check.error('Check URL status is: %s', status)
            validFlag = False
    except:
        log_check.error ("error: ", sys.exc_info()[0])
        validFlag = False
    finally:
        return validFlag


def check_md5(origin_source_file, download_source_file, source_index):
    """
    Check the md5 of two source files. 
    """
    m1 = get_md5(origin_source_file)
    m2 = get_md5(download_source_file)
    if m1 != m2:
        return False
    else:
        return True


def get_md5(file):
    """
    Get the md5 of given file. 
    """
    md5 = hashlib.md5()
    f = open(file, 'rb')
    while True:
        b = f.read(8096)
        if not b:
            break
        md5.update(b)
    f.close()
    return md5.hexdigest()


def truncate_file(file):
    """
    Truncate the given file. 
    """
    if not os.path.exists(file):
        os.system(r'touch %s' % file)
    with open(file, 'r+') as f:
        f.truncate()


def get_http_status_code(url):
    """
    Get the http status code of given url. 
    """
    validOfhttp = url.startswith( 'http:' );
    validOfhttps = url.startswith( 'https:' );
    httpStatusCode = 404
    
    if validOfhttp == True or validOfhttps == True:
        try:
            request = requests.get(url)
            httpStatusCode = request.status_code
        except Exception as ex:
            template = "An exception of type {0} occurred. Arguments:\n{1!r}"
            message = template.format(type(ex).__name__, ex.args)
            log_check.error(message)
        finally:
            return httpStatusCode


def get_contents(filename):
    """
    Get file contents and return values. If read failed return None.
    """
    with open(filename, "rb") as f:
        return f.read()
    return None


def extract_tar(tarball_path, extract_path):
    """
    Extract tar package in extract_path. If extract failed the program will exit.
    """
    res_flag = True
    if not os.path.isfile(tarball_path):
        log_check.error("%s is not a file", tarball_path)
        res_flag = False
        return res_flag

    try:
        with tarfile.open(tarball_path) as content:
            content.extractall(path=extract_path)
    except FileNotFoundError:
        res_flag = False
        log_check.error("%s can not be found", tarball_path)
    except tarfile.CompressionError:
        res_flag = False
        log_check.error("%s can not be decoded correctly", tarball_path)
    except tarfile.ReadError:
        res_flag = False
        log_check.error("%s is invalid tar file", tarball_path)
    finally:
        return res_flag


def extract_bz2(bz2_path, extract_path):
    """
    Extract bz2 package in extract_path. If extract failed the program will exit.
    """
    res_flag = True
    if not os.path.isfile(bz2_path):
        log_check.error("%s is not a bz2 file", bz2_path)
        res_flag = False
        return res_flag
        
    try:
        archive = tarfile.open(bz2_path,'r:bz2')
        archive.debug = 1    # Display the files beeing decompressed.
        for tarinfo in archive:
            archive.extract(tarinfo, extract_path) # d:\mydirectory is where I want to uncompress the files.
        archive.close()
    except FileNotFoundError:
        res_flag = False
        log_check.error("%s can not be found", bz2_path)
    except tarfile.CompressionError:
        res_flag = False
        log_check.error("%s can not be decoded correctly", tarball_path)
    except tarfile.ReadError:
        res_flag = False
        log_check.error("%s is invalid tar file", tarball_path)
    finally:
        return res_flag

def extract_zip(zip_path, extract_path):
    """
    Extract zip package in extract_path. If extract failed the program will exit.
    """
    res_flag = True
    if not os.path.isfile(zip_path):
        log_check.error("%s is not a zip file", zip_path)
        res_flag = False
        return res_flag

    try:
        zip_file = zipfile.ZipFile(zip_path)
        zip_file.extractall(extract_path)
        zip_file.close()
    except FileNotFoundError:
        res_flag = False
        log_check.error("%s can not be found", zip_path)
    except zipfile.BadZipfile:
        res_flag = False
        log_check.error("%s is bad zip file", zip_path)
    except zipfile.LargeZipFile:
        res_flag = False
        log_check.error("The zip file requires the zip64 feature but is not enabled")
    finally:
        return res_flag


def decode_license(license_string, charset):
    """ 
    Decode the license string. return the license string or nothing.
    """
    if not charset:
        return
    return license_string.decode(charset)


def add_license_from_spec_file(license_string):
    """
    Add license to licenses_for_spec.
    """
    if license_string not in licenses_for_spec:
        licenses_for_spec.append(license_string)


def add_license_from_license_file(license_string):
    """
    Add license to licenses_for_source_files.
    """
    if license_string not in licenses_for_source_files:
        licenses_for_source_files.append(license_string)


def scan_licenses(copying):
    """
    Scan licenses from copying file and add to licenses_for_source_files.
    if get contents failed or decode data failed, return nothing.
    """
    try:
        data = get_contents(copying)
    except FileNotFoundError:
        return
    data = decode_license(data, chardet.detect(data)['encoding'])
    if not data:
        return 
    for word in license_translations:
        if word in data:
            real_word = license_translations.get(word, word)
            add_license_from_license_file(real_word)


def clean_license_string(lic):
    """
    Clean up license string by replace substrings.
    """
    reps = [(" (", "("),
            (" v2", "-2"),
            (" v3", "-3"),
            (" 2", "-2"),
            (" 3", "-3"),
            (" <", "<"),
            (" >", ">"),
            ("= ", "="),
            ("GPL(>=-2)", "GPL-2.0+"),
            ("Modified", ""),
            ("OSI", ""),
            ("Approved", ""),
            ("Simplified", ""),
            ("file", ""),
            ("LICENSE", "")]

    for sub, rep in reps:
        lic = lic.replace(sub, rep)
    return lic


def scan_licenses_in_SPEC(specfile):
    """
    Find spec file and scan. If no spec file or open file failed, the program will exit with an error. 
    """
    s_spec = Spec.from_file(specfile)
    licenses = replace_macros(s_spec.license, s_spec)

    if licenses in license_translations:
        real_words = license_translations.get(licenses, licenses)
        add_license_from_spec_file(real_words)
    else:
        words = split_license(licenses)
        for word in words:
            real_word = license_translations.get(word, word)
            log_check.debug("after translate license_string ==> "
                        "real_license: %s ==> %s", word, real_word)
            add_license_from_spec_file(real_word)                       
    log_check.info("all licenses from SPEC: %s", licenses_for_spec)


def split_license(licenses):
    """
    分割spec license字段的license
    """
    license_set = re.split("r'/\s?|\(|\)|\,|[Aa][Nn][Dd]|or|OR|\s?/g'", licenses)
    for index in range(len(license_set)): # 去除字符串首尾空格
        license_set[index] = license_set[index].strip()
    license_set = list(filter(None, license_set)) # 去除list中空字符串
    return license_set


def check_licenses_is_same():
    """
    Check if the licenses from SPEC is the same as the licenses from LICENSE file.
    if same, return True. if not same return False.
    """
    for lic_from_licenses in licenses_for_source_files:
        if lic_from_licenses not in licenses_for_spec:
            return False 
    for lic_from_spec in licenses_for_spec:
        if lic_from_spec not in licenses_for_source_files:
            return False
    return True


def overwrite_spec(specfile):
    """
    Write License in SPEC file.
    If open file failed, return nothing.
    """
    licenses_for_wirte = "License:\t"
    for lic in licenses_for_source_files:
        if lic == licenses_for_source_files[0]:
            licenses_for_wirte += lic
        else:
            licenses_for_wirte += " and " + lic
    licenses_for_wirte += "\n"
    
    try:
        with open(specfile, 'r') as specfd:
            lines = specfd.readlines()
            specfd.close()
    except FileNotFoundError:
        return
    f = open(specfile, 'w')
    for line in lines:
        if line.startswith("License"):
            f.write(licenses_for_wirte)
        else:
            f.write(line)
    f.close()
    log_check.info("licenses wirte to spec success")


def change_spec_by_shell(shell, filepath):
    """
    Format the given spec file.
    """
    # bash path/spec_prep_plus.sh path/package_name/*.spec
    cmd = "bash " + shell + " " + filepath + "/*.spec"
    try:
        proc_new = subprocess.Popen(cmd, stdout = subprocess.PIPE, shell = True)
    except OSError as err:
        if err.errno == errno.ENOENT:
            logger.error("Command %s not found." % cmd)
        return None, None
    stdout_new, stderr_new = proc_new.communicate()
    returncode = proc_new.returncode
    proc_new.poll()
    return stdout_new.decode('utf-8', 'ignore'), returncode


def process_log(logFilename):
    """
    Extract clear error information from the full log.
    """
    with open(logFilename, 'r') as file_to_read:
        with open('packages_errorLog.txt', 'w') as errorLog:
            for line in file_to_read.readlines():
                if('*******' in line or "start process package" in line or "ERROR" in line):
                    if('exc_type' in line):
                        errorLog.write(preLine)
                    errorLog.write(line)
                preLine = line


def record_pkg_List():
    """
    Extract the list of software packages that passed 
    and failed inspections according to the logs.
    """
    NOTOK_pkg_list.sort()
    OK_pkg_list.sort()
    
    with open ("NOTOK_pkg_list.txt",'a') as f:
        f.write("\n".join(NOTOK_pkg_list))
        
    with open ("OK_pkg_list.txt",'a') as f:
        f.write("\n".join(OK_pkg_list))


def is_spec_file(filename):
    """
    根据后缀名判断 是否为spec文件
    :return bool
    """
    return filename.endswith(".spec")


def is_unused_spec_file(filename):
    """
    根据后缀名判断 非软件包必须文件
    :return bool
    """
    return filename.endswith(UNUSED_FILE_SUFFIX)


if __name__ == '__main__':
    main()
